{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "      <th>E</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2013-01-01</th>\n",
       "      <td>1.063993</td>\n",
       "      <td>1.076246</td>\n",
       "      <td>0.304555</td>\n",
       "      <td>-0.790627</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-02</th>\n",
       "      <td>-0.055015</td>\n",
       "      <td>-0.486989</td>\n",
       "      <td>0.205795</td>\n",
       "      <td>0.402057</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-03</th>\n",
       "      <td>0.988586</td>\n",
       "      <td>-0.285283</td>\n",
       "      <td>0.236512</td>\n",
       "      <td>0.804668</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-04</th>\n",
       "      <td>0.435068</td>\n",
       "      <td>0.973593</td>\n",
       "      <td>-0.519439</td>\n",
       "      <td>0.512159</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   A         B         C         D    E\n",
       "2013-01-01  1.063993  1.076246  0.304555 -0.790627  1.0\n",
       "2013-01-02 -0.055015 -0.486989  0.205795  0.402057  1.0\n",
       "2013-01-03  0.988586 -0.285283  0.236512  0.804668  NaN\n",
       "2013-01-04  0.435068  0.973593 -0.519439  0.512159  NaN"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "dates = pd.date_range('20130101', periods=6)\n",
    "df = pd.DataFrame(np.random.randn(6,4),index=dates,columns=list('ABCD'))\n",
    "df1 = df.reindex(index=dates[0:4],columns=list(df.columns)+['E'])\n",
    "df1.loc[dates[0]:dates[1],'E']=1\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "      <th>E</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2013-01-01</th>\n",
       "      <td>1.063993</td>\n",
       "      <td>1.076246</td>\n",
       "      <td>0.304555</td>\n",
       "      <td>-0.790627</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-02</th>\n",
       "      <td>-0.055015</td>\n",
       "      <td>-0.486989</td>\n",
       "      <td>0.205795</td>\n",
       "      <td>0.402057</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   A         B         C         D    E\n",
       "2013-01-01  1.063993  1.076246  0.304555 -0.790627  1.0\n",
       "2013-01-02 -0.055015 -0.486989  0.205795  0.402057  1.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#删除包含你缺失的行\n",
    "df1.dropna(how='any')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "      <th>E</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2013-01-01</th>\n",
       "      <td>1.063993</td>\n",
       "      <td>1.076246</td>\n",
       "      <td>0.304555</td>\n",
       "      <td>-0.790627</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-02</th>\n",
       "      <td>-0.055015</td>\n",
       "      <td>-0.486989</td>\n",
       "      <td>0.205795</td>\n",
       "      <td>0.402057</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-03</th>\n",
       "      <td>0.988586</td>\n",
       "      <td>-0.285283</td>\n",
       "      <td>0.236512</td>\n",
       "      <td>0.804668</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-04</th>\n",
       "      <td>0.435068</td>\n",
       "      <td>0.973593</td>\n",
       "      <td>-0.519439</td>\n",
       "      <td>0.512159</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   A         B         C         D    E\n",
       "2013-01-01  1.063993  1.076246  0.304555 -0.790627  1.0\n",
       "2013-01-02 -0.055015 -0.486989  0.205795  0.402057  1.0\n",
       "2013-01-03  0.988586 -0.285283  0.236512  0.804668  5.0\n",
       "2013-01-04  0.435068  0.973593 -0.519439  0.512159  5.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#对缺失值进行填充\n",
    "df1.fillna(value=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "      <th>E</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2013-01-01</th>\n",
       "      <td>1.063993</td>\n",
       "      <td>1.076246</td>\n",
       "      <td>0.304555</td>\n",
       "      <td>-0.790627</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-02</th>\n",
       "      <td>-0.055015</td>\n",
       "      <td>-0.486989</td>\n",
       "      <td>0.205795</td>\n",
       "      <td>0.402057</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-03</th>\n",
       "      <td>0.988586</td>\n",
       "      <td>-0.285283</td>\n",
       "      <td>0.236512</td>\n",
       "      <td>0.804668</td>\n",
       "      <td>0.09042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-04</th>\n",
       "      <td>0.435068</td>\n",
       "      <td>0.973593</td>\n",
       "      <td>-0.519439</td>\n",
       "      <td>0.512159</td>\n",
       "      <td>0.09042</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   A         B         C         D        E\n",
       "2013-01-01  1.063993  1.076246  0.304555 -0.790627  1.00000\n",
       "2013-01-02 -0.055015 -0.486989  0.205795  0.402057  1.00000\n",
       "2013-01-03  0.988586 -0.285283  0.236512  0.804668  0.09042\n",
       "2013-01-04  0.435068  0.973593 -0.519439  0.512159  0.09042"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.fillna(df['A'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'louwill is a machine learning engineer.'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 去除空格\n",
    "char = '    louwill is a machine learning engineer.    '\n",
    "char.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['louwill', 'is', 'a', 'machine', 'learning', 'engineer.']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 字符串分割\n",
    "char2 = 'louwill,is,a,machine,learning,engineer.'\n",
    "char2.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'louwill,is,a,machine,learning,engineer.'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 拼接：列表转字符串\n",
    "char2_2 = char2.split(',')\n",
    "','.join(char2_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'louwill is a machine learning engineer.'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 字符替换\n",
    "char2.replace(',', ' ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 正则表达式 \n",
    "- re模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['good', 'cool']\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "# compile函数：编译正则表达式模式\n",
    "# re.compile(pattern, flag=0)\n",
    "text1 = 'lebron is a slight good person, he is cool.'\n",
    "rr = re.compile(r'\\w*oo\\w*')\n",
    "print(rr.findall(text1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "com\n"
     ]
    }
   ],
   "source": [
    "# match函数：从字符串首开始匹配\n",
    "# re.match(pattern, string, flag=0)\n",
    "print(re.match('com', 'com.louwill.con').group())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# search函数：对字符串启动查找模式进行匹配，找到第一个并返回\n",
    "# re.search(pattern, string, flags=0)\n",
    "# sub 函数：替换字符串中每一个匹配的子串并返回替换后的字符串\n",
    "# re.sub(pattern, repl, string, count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 招聘数据的清洗过程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1650, 14)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "data1 = pd.read_csv('./data_analysis.csv', encoding='gbk')\n",
    "data2 = pd.read_csv('./machine_learning.csv', encoding='gbk')\n",
    "data3 = pd.read_csv('./data_mining.csv', encoding='gbk')\n",
    "data4 = pd.read_csv('./deep_learning.csv', encoding='gbk')\n",
    "\n",
    "data = pd.concat((pd.concat((pd.concat((data1, data2)), data3)), data4)).reset_index(drop=True)\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>address</th>\n",
       "      <th>advantage</th>\n",
       "      <th>city</th>\n",
       "      <th>company_name</th>\n",
       "      <th>education</th>\n",
       "      <th>industry</th>\n",
       "      <th>industryLables</th>\n",
       "      <th>label</th>\n",
       "      <th>position_detail</th>\n",
       "      <th>position_name</th>\n",
       "      <th>salary</th>\n",
       "      <th>size</th>\n",
       "      <th>stage</th>\n",
       "      <th>work_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>['科华北路', '桂溪', '四川大学']</td>\n",
       "      <td>工作氛围好</td>\n",
       "      <td>成都</td>\n",
       "      <td>达疆网络科技（上海）有限公司</td>\n",
       "      <td>本科</td>\n",
       "      <td>O2O</td>\n",
       "      <td>[]</td>\n",
       "      <td>['年底双薪', '绩效奖金', '岗位晋升', '定期体检']</td>\n",
       "      <td>\\r职位描述：\\r\\r工作职责：?\\r1、负责新零售业务的数据分析工作，挖掘数据分析需求，制...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10k-20k</td>\n",
       "      <td>2000人以上</td>\n",
       "      <td>D轮及以上</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>六险二金,晋升通道,独当一面,话语权</td>\n",
       "      <td>北京</td>\n",
       "      <td>贝壳找房（北京）科技有限公司</td>\n",
       "      <td>本科</td>\n",
       "      <td>移动互联网,O2O</td>\n",
       "      <td>['大数据', '商业']</td>\n",
       "      <td>['股票期权', '带薪年假', '绩效奖金', '扁平管理']</td>\n",
       "      <td>\\r职位描述：\\r\\r工作职责:方向一、经营分析/指标体系1. 参与公司核心策略的数据分析，...</td>\n",
       "      <td>数据分析类</td>\n",
       "      <td>25k-40k</td>\n",
       "      <td>2000人以上</td>\n",
       "      <td>C轮</td>\n",
       "      <td>5-10年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>['中关村', '万泉河', '苏州街']</td>\n",
       "      <td>五险一金</td>\n",
       "      <td>北京</td>\n",
       "      <td>杭州财米科技有限公司</td>\n",
       "      <td>不限</td>\n",
       "      <td>移动互联网,金融</td>\n",
       "      <td>[]</td>\n",
       "      <td>['年底多薪', '岗位晋升', '定期体检', '五险一金']</td>\n",
       "      <td>\\r职位描述：\\r\\r职位描述：\\r1、收集、处理用户海量数据，挖掘用户行为特征，为产品、运...</td>\n",
       "      <td>数据分析师 (MJ000766)</td>\n",
       "      <td>10k-15k</td>\n",
       "      <td>500-2000人</td>\n",
       "      <td>C轮</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>['琶洲', '官洲']</td>\n",
       "      <td>六险一金,周末双休,营养工作餐,萌宠陪伴</td>\n",
       "      <td>广州</td>\n",
       "      <td>广州微咔世纪信息科技有限公司</td>\n",
       "      <td>大专</td>\n",
       "      <td>移动互联网,社交网络</td>\n",
       "      <td>['移动互联网', '社交', '数据运营']</td>\n",
       "      <td>['六险一金', '周末双休', '营养工作餐', '暖心下午茶']</td>\n",
       "      <td>\\r职位描述：\\r\\r职位描述：\\r1.负责对业务的专题进行统计分析，形成专题分析报告；\\r...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8k-15k</td>\n",
       "      <td>50-150人</td>\n",
       "      <td>A轮</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>['中关村', '北京大学', '颐和园']</td>\n",
       "      <td>爱奇艺</td>\n",
       "      <td>北京</td>\n",
       "      <td>北京奇艺世纪科技有限公司</td>\n",
       "      <td>本科</td>\n",
       "      <td>广告营销,文化娱乐</td>\n",
       "      <td>['大数据', 'SPSS']</td>\n",
       "      <td>['绩效奖金', '五险一金', '交通补助', '带薪年假']</td>\n",
       "      <td>\\r职位描述：\\r\\r岗位职责： 1、负责内容合作部产品及运营数据指标的搭建； 2、负责数据...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8k-12k</td>\n",
       "      <td>2000人以上</td>\n",
       "      <td>上市公司</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  address             advantage city    company_name  \\\n",
       "0  ['科华北路', '桂溪', '四川大学']                 工作氛围好   成都  达疆网络科技（上海）有限公司   \n",
       "1                     NaN    六险二金,晋升通道,独当一面,话语权   北京  贝壳找房（北京）科技有限公司   \n",
       "2   ['中关村', '万泉河', '苏州街']                  五险一金   北京      杭州财米科技有限公司   \n",
       "3            ['琶洲', '官洲']  六险一金,周末双休,营养工作餐,萌宠陪伴   广州  广州微咔世纪信息科技有限公司   \n",
       "4  ['中关村', '北京大学', '颐和园']                   爱奇艺   北京    北京奇艺世纪科技有限公司   \n",
       "\n",
       "  education    industry           industryLables  \\\n",
       "0        本科         O2O                       []   \n",
       "1        本科   移动互联网,O2O            ['大数据', '商业']   \n",
       "2        不限    移动互联网,金融                       []   \n",
       "3        大专  移动互联网,社交网络  ['移动互联网', '社交', '数据运营']   \n",
       "4        本科   广告营销,文化娱乐          ['大数据', 'SPSS']   \n",
       "\n",
       "                                label  \\\n",
       "0    ['年底双薪', '绩效奖金', '岗位晋升', '定期体检']   \n",
       "1    ['股票期权', '带薪年假', '绩效奖金', '扁平管理']   \n",
       "2    ['年底多薪', '岗位晋升', '定期体检', '五险一金']   \n",
       "3  ['六险一金', '周末双休', '营养工作餐', '暖心下午茶']   \n",
       "4    ['绩效奖金', '五险一金', '交通补助', '带薪年假']   \n",
       "\n",
       "                                     position_detail     position_name  \\\n",
       "0  \\r职位描述：\\r\\r工作职责：?\\r1、负责新零售业务的数据分析工作，挖掘数据分析需求，制...             数据分析师   \n",
       "1  \\r职位描述：\\r\\r工作职责:方向一、经营分析/指标体系1. 参与公司核心策略的数据分析，...             数据分析类   \n",
       "2  \\r职位描述：\\r\\r职位描述：\\r1、收集、处理用户海量数据，挖掘用户行为特征，为产品、运...  数据分析师 (MJ000766)   \n",
       "3  \\r职位描述：\\r\\r职位描述：\\r1.负责对业务的专题进行统计分析，形成专题分析报告；\\r...             数据分析师   \n",
       "4  \\r职位描述：\\r\\r岗位职责： 1、负责内容合作部产品及运营数据指标的搭建； 2、负责数据...             数据分析师   \n",
       "\n",
       "    salary       size  stage work_year  \n",
       "0  10k-20k    2000人以上  D轮及以上      1-3年  \n",
       "1  25k-40k    2000人以上     C轮     5-10年  \n",
       "2  10k-15k  500-2000人     C轮      1-3年  \n",
       "3   8k-15k    50-150人     A轮      1-3年  \n",
       "4   8k-12k    2000人以上   上市公司      1-3年  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1650 entries, 0 to 1649\n",
      "Data columns (total 14 columns):\n",
      "address            1074 non-null object\n",
      "advantage          1650 non-null object\n",
      "city               1650 non-null object\n",
      "company_name       1650 non-null object\n",
      "education          1650 non-null object\n",
      "industry           1650 non-null object\n",
      "industryLables     1650 non-null object\n",
      "label              1650 non-null object\n",
      "position_detail    1646 non-null object\n",
      "position_name      1650 non-null object\n",
      "salary             1650 non-null object\n",
      "size               1650 non-null object\n",
      "stage              1650 non-null object\n",
      "work_year          1650 non-null object\n",
      "dtypes: object(14)\n",
      "memory usage: 180.5+ KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    ['科华北路', '桂溪', '四川大学']\n",
       "1                    ['未知']\n",
       "2     ['中关村', '万泉河', '苏州街']\n",
       "3              ['琶洲', '官洲']\n",
       "4    ['中关村', '北京大学', '颐和园']\n",
       "Name: address, dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['address'] = data['address'].fillna(\"['未知']\")\n",
    "data['address'][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    '科华北路', '桂溪', '四川大学'\n",
       "1                    '未知'\n",
       "2     '中关村', '万泉河', '苏州街'\n",
       "3              '琶洲', '官洲'\n",
       "4    '中关村', '北京大学', '颐和园'\n",
       "Name: address, dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i, j in enumerate(data['address']):\n",
    "    j = j.replace('[', '').replace(']', '')\n",
    "    data['address'][i] = j\n",
    "    \n",
    "data['address'][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                         \n",
       "1              '大数据', '商业'\n",
       "2                         \n",
       "3    '移动互联网', '社交', '数据运营'\n",
       "4            '大数据', 'SPSS'\n",
       "5            '大数据', '数据挖掘'\n",
       "6            '大数据', '数据挖掘'\n",
       "7            '大数据', '数据挖掘'\n",
       "8                     '金融'\n",
       "9                         \n",
       "Name: industryLables, dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i, j in enumerate(data['industryLables']):\n",
    "    j = j.replace('[', '').replace(']', '')\n",
    "    data['industryLables'][i] = j\n",
    "    \n",
    "data['industryLables'][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      '年底双薪', '绩效奖金', '岗位晋升', '定期体检'\n",
       "1      '股票期权', '带薪年假', '绩效奖金', '扁平管理'\n",
       "2      '年底多薪', '岗位晋升', '定期体检', '五险一金'\n",
       "3    '六险一金', '周末双休', '营养工作餐', '暖心下午茶'\n",
       "4      '绩效奖金', '五险一金', '交通补助', '带薪年假'\n",
       "5      '专项奖金', '带薪年假', '弹性工作', '管理规范'\n",
       "6        '丰厚年终', '扁平管理', '追求极致', '本分'\n",
       "7      '弹性工作', '五险一金', '年度旅游', '年底双薪'\n",
       "8      '创新开放', '团队牛X', '全员期权', '高速成长'\n",
       "9      '绩效奖金', '交通补助', '午餐补助', '定期体检'\n",
       "Name: label, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i, j in enumerate(data['label']):\n",
    "    j = j.replace('[', '').replace(']', '')\n",
    "    data['label'][i] = j\n",
    "    \n",
    "data['label'][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'职位描述：工作职责：?1、负责新零售业务的数据分析工作，挖掘数据分析需求，制定并实施分析方案，并根据数据分析结果为业务的改进提出合理化建议；?2、通过专题分析，对业务问题进行深入分析，为产品改进、运营决策、营销推广策略提供数据支持，推动业务部门数据驱动业务决策的转化3、通过对公司运营数据研究，提出改善运营质量的方法和建议，搭建BI数据分析体系，为公司决策提供支持。任职资格：?1、统计、数学、信息技术相关专业本科以上学历，互联网2年以上数据分析/挖掘相关经验；2、熟练独立编写商业数据分析报告，及时发现和分析其中隐含的变化和问题；?3、良好的数据敏感度，能从海量数据提炼核心结果；有丰富的数据分析、挖掘、清洗和建模的经验；?4、思维敏捷，具有发散性，能够举一反三，良好的逻辑分析能力及问题解决能力5、良好的跨团队、部门沟通及推动能力，有强烈的主人翁意识，积极发扬团队合作精神6、 熟练掌握SQL7、熟悉或使用过至少一种统计分析/数据挖掘软件（R,?Python等）者优先8、有大数据处理经验，如Hive/Spark/Hadoop等使用经验者优先'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['position_detail'][0].replace('\\r', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['position_detail'] = data['position_detail'].fillna('未知')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    职位描述：工作职责：?1、负责新零售业务的数据分析工作，挖掘数据分析需求，制定并实施分析方案...\n",
       "1    职位描述：工作职责:方向一、经营分析/指标体系1. 参与公司核心策略的数据分析，基于策略逻辑...\n",
       "2    职位描述：职位描述：1、收集、处理用户海量数据，挖掘用户行为特征，为产品、运营提供参考依据；...\n",
       "Name: position_detail, dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i, j in enumerate(data['position_detail']):\n",
    "    j = j.replace('\\r', '')\n",
    "    data['position_detail'][i] = j\n",
    "    \n",
    "data['position_detail'][:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>address</th>\n",
       "      <th>advantage</th>\n",
       "      <th>city</th>\n",
       "      <th>company_name</th>\n",
       "      <th>education</th>\n",
       "      <th>industry</th>\n",
       "      <th>industryLables</th>\n",
       "      <th>label</th>\n",
       "      <th>position_detail</th>\n",
       "      <th>position_name</th>\n",
       "      <th>salary</th>\n",
       "      <th>size</th>\n",
       "      <th>stage</th>\n",
       "      <th>work_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>'科华北路', '桂溪', '四川大学'</td>\n",
       "      <td>工作氛围好</td>\n",
       "      <td>成都</td>\n",
       "      <td>达疆网络科技（上海）有限公司</td>\n",
       "      <td>本科</td>\n",
       "      <td>O2O</td>\n",
       "      <td></td>\n",
       "      <td>'年底双薪', '绩效奖金', '岗位晋升', '定期体检'</td>\n",
       "      <td>职位描述：工作职责：?1、负责新零售业务的数据分析工作，挖掘数据分析需求，制定并实施分析方案...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10k-20k</td>\n",
       "      <td>2000人以上</td>\n",
       "      <td>D轮及以上</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'未知'</td>\n",
       "      <td>六险二金,晋升通道,独当一面,话语权</td>\n",
       "      <td>北京</td>\n",
       "      <td>贝壳找房（北京）科技有限公司</td>\n",
       "      <td>本科</td>\n",
       "      <td>移动互联网,O2O</td>\n",
       "      <td>'大数据', '商业'</td>\n",
       "      <td>'股票期权', '带薪年假', '绩效奖金', '扁平管理'</td>\n",
       "      <td>职位描述：工作职责:方向一、经营分析/指标体系1. 参与公司核心策略的数据分析，基于策略逻辑...</td>\n",
       "      <td>数据分析类</td>\n",
       "      <td>25k-40k</td>\n",
       "      <td>2000人以上</td>\n",
       "      <td>C轮</td>\n",
       "      <td>5-10年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>'中关村', '万泉河', '苏州街'</td>\n",
       "      <td>五险一金</td>\n",
       "      <td>北京</td>\n",
       "      <td>杭州财米科技有限公司</td>\n",
       "      <td>不限</td>\n",
       "      <td>移动互联网,金融</td>\n",
       "      <td></td>\n",
       "      <td>'年底多薪', '岗位晋升', '定期体检', '五险一金'</td>\n",
       "      <td>职位描述：职位描述：1、收集、处理用户海量数据，挖掘用户行为特征，为产品、运营提供参考依据；...</td>\n",
       "      <td>数据分析师 (MJ000766)</td>\n",
       "      <td>10k-15k</td>\n",
       "      <td>500-2000人</td>\n",
       "      <td>C轮</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>'琶洲', '官洲'</td>\n",
       "      <td>六险一金,周末双休,营养工作餐,萌宠陪伴</td>\n",
       "      <td>广州</td>\n",
       "      <td>广州微咔世纪信息科技有限公司</td>\n",
       "      <td>大专</td>\n",
       "      <td>移动互联网,社交网络</td>\n",
       "      <td>'移动互联网', '社交', '数据运营'</td>\n",
       "      <td>'六险一金', '周末双休', '营养工作餐', '暖心下午茶'</td>\n",
       "      <td>职位描述：职位描述：1.负责对业务的专题进行统计分析，形成专题分析报告；2.基于业务理解，设...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8k-15k</td>\n",
       "      <td>50-150人</td>\n",
       "      <td>A轮</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>'中关村', '北京大学', '颐和园'</td>\n",
       "      <td>爱奇艺</td>\n",
       "      <td>北京</td>\n",
       "      <td>北京奇艺世纪科技有限公司</td>\n",
       "      <td>本科</td>\n",
       "      <td>广告营销,文化娱乐</td>\n",
       "      <td>'大数据', 'SPSS'</td>\n",
       "      <td>'绩效奖金', '五险一金', '交通补助', '带薪年假'</td>\n",
       "      <td>职位描述：岗位职责： 1、负责内容合作部产品及运营数据指标的搭建； 2、负责数据后台的完善与...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8k-12k</td>\n",
       "      <td>2000人以上</td>\n",
       "      <td>上市公司</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                address             advantage city    company_name education  \\\n",
       "0  '科华北路', '桂溪', '四川大学'                 工作氛围好   成都  达疆网络科技（上海）有限公司        本科   \n",
       "1                  '未知'    六险二金,晋升通道,独当一面,话语权   北京  贝壳找房（北京）科技有限公司        本科   \n",
       "2   '中关村', '万泉河', '苏州街'                  五险一金   北京      杭州财米科技有限公司        不限   \n",
       "3            '琶洲', '官洲'  六险一金,周末双休,营养工作餐,萌宠陪伴   广州  广州微咔世纪信息科技有限公司        大专   \n",
       "4  '中关村', '北京大学', '颐和园'                   爱奇艺   北京    北京奇艺世纪科技有限公司        本科   \n",
       "\n",
       "     industry         industryLables                             label  \\\n",
       "0         O2O                           '年底双薪', '绩效奖金', '岗位晋升', '定期体检'   \n",
       "1   移动互联网,O2O            '大数据', '商业'    '股票期权', '带薪年假', '绩效奖金', '扁平管理'   \n",
       "2    移动互联网,金融                           '年底多薪', '岗位晋升', '定期体检', '五险一金'   \n",
       "3  移动互联网,社交网络  '移动互联网', '社交', '数据运营'  '六险一金', '周末双休', '营养工作餐', '暖心下午茶'   \n",
       "4   广告营销,文化娱乐          '大数据', 'SPSS'    '绩效奖金', '五险一金', '交通补助', '带薪年假'   \n",
       "\n",
       "                                     position_detail     position_name  \\\n",
       "0  职位描述：工作职责：?1、负责新零售业务的数据分析工作，挖掘数据分析需求，制定并实施分析方案...             数据分析师   \n",
       "1  职位描述：工作职责:方向一、经营分析/指标体系1. 参与公司核心策略的数据分析，基于策略逻辑...             数据分析类   \n",
       "2  职位描述：职位描述：1、收集、处理用户海量数据，挖掘用户行为特征，为产品、运营提供参考依据；...  数据分析师 (MJ000766)   \n",
       "3  职位描述：职位描述：1.负责对业务的专题进行统计分析，形成专题分析报告；2.基于业务理解，设...             数据分析师   \n",
       "4  职位描述：岗位职责： 1、负责内容合作部产品及运营数据指标的搭建； 2、负责数据后台的完善与...             数据分析师   \n",
       "\n",
       "    salary       size  stage work_year  \n",
       "0  10k-20k    2000人以上  D轮及以上      1-3年  \n",
       "1  25k-40k    2000人以上     C轮     5-10年  \n",
       "2  10k-15k  500-2000人     C轮      1-3年  \n",
       "3   8k-15k    50-150人     A轮      1-3年  \n",
       "4   8k-12k    2000人以上   上市公司      1-3年  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15.0\n",
      "32.5\n",
      "12.5\n",
      "11.5\n",
      "10.0\n",
      "30.0\n",
      "22.5\n",
      "25.0\n",
      "22.5\n",
      "9.0\n"
     ]
    }
   ],
   "source": [
    "import string\n",
    "for i in data['salary'][:10]:\n",
    "    i = i.replace('k', '')\n",
    "    i1 = int(i.split('-')[0])\n",
    "    i2 = int(i.split('-')[1])\n",
    "    i3 = 1/2 * (i1+i2)\n",
    "    print(i3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    15000\n",
       "1    32500\n",
       "2    12500\n",
       "3    11500\n",
       "4    10000\n",
       "5    30000\n",
       "6    22500\n",
       "7    25000\n",
       "8    22500\n",
       "9     9000\n",
       "Name: salary, dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i, j in enumerate(data['salary']):\n",
    "    j = j.replace('k', '').replace('K', '').replace('以上', '-0')\n",
    "    j1 = int(j.split('-')[0])\n",
    "    j2 = int(j.split('-')[1])\n",
    "    j3 = 1/2 * (j1+j2)\n",
    "    data['salary'][i] = j3*1000\n",
    "    \n",
    "data['salary'].head(10)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2000人以上      573\n",
       "500-2000人    324\n",
       "150-500人     314\n",
       "50-150人      269\n",
       "15-50人       156\n",
       "少于15人         14\n",
       "Name: size, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['size'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "上市公司     337\n",
       "不需要融资    293\n",
       "B轮       240\n",
       "A轮       239\n",
       "C轮       216\n",
       "D轮及以上    192\n",
       "天使轮       69\n",
       "未融资       64\n",
       "Name: stage, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['stage'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3-5年     730\n",
       "1-3年     465\n",
       "不限       221\n",
       "5-10年    136\n",
       "应届毕业生     80\n",
       "1年以下      16\n",
       "10年以上      2\n",
       "Name: work_year, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['work_year'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    数据分析师\n",
       "1    数据分析师\n",
       "2    数据分析师\n",
       "3    数据分析师\n",
       "4    数据分析师\n",
       "Name: position_name, dtype: object"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i, j in enumerate(data['position_name']):\n",
    "    if '数据分析' in j:\n",
    "        j = '数据分析师'\n",
    "    if '数据挖掘' in j:\n",
    "        j = '数据挖掘工程师'\n",
    "    if '机器学习' in j:\n",
    "        j = '机器学习工程师'\n",
    "    if '深度学习' in j:\n",
    "        j = '深度学习工程师'\n",
    "    data['position_name'][i] = j\n",
    "data['position_name'][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>address</th>\n",
       "      <th>advantage</th>\n",
       "      <th>city</th>\n",
       "      <th>company_name</th>\n",
       "      <th>education</th>\n",
       "      <th>industry</th>\n",
       "      <th>industryLables</th>\n",
       "      <th>label</th>\n",
       "      <th>position_detail</th>\n",
       "      <th>position_name</th>\n",
       "      <th>salary</th>\n",
       "      <th>size</th>\n",
       "      <th>stage</th>\n",
       "      <th>work_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>'科华北路', '桂溪', '四川大学'</td>\n",
       "      <td>工作氛围好</td>\n",
       "      <td>成都</td>\n",
       "      <td>达疆网络科技（上海）有限公司</td>\n",
       "      <td>本科</td>\n",
       "      <td>O2O</td>\n",
       "      <td></td>\n",
       "      <td>'年底双薪', '绩效奖金', '岗位晋升', '定期体检'</td>\n",
       "      <td>职位描述：工作职责：?1、负责新零售业务的数据分析工作，挖掘数据分析需求，制定并实施分析方案...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>15000</td>\n",
       "      <td>2000人以上</td>\n",
       "      <td>D轮及以上</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'未知'</td>\n",
       "      <td>六险二金,晋升通道,独当一面,话语权</td>\n",
       "      <td>北京</td>\n",
       "      <td>贝壳找房（北京）科技有限公司</td>\n",
       "      <td>本科</td>\n",
       "      <td>移动互联网,O2O</td>\n",
       "      <td>'大数据', '商业'</td>\n",
       "      <td>'股票期权', '带薪年假', '绩效奖金', '扁平管理'</td>\n",
       "      <td>职位描述：工作职责:方向一、经营分析/指标体系1. 参与公司核心策略的数据分析，基于策略逻辑...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>32500</td>\n",
       "      <td>2000人以上</td>\n",
       "      <td>C轮</td>\n",
       "      <td>5-10年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>'中关村', '万泉河', '苏州街'</td>\n",
       "      <td>五险一金</td>\n",
       "      <td>北京</td>\n",
       "      <td>杭州财米科技有限公司</td>\n",
       "      <td>不限</td>\n",
       "      <td>移动互联网,金融</td>\n",
       "      <td></td>\n",
       "      <td>'年底多薪', '岗位晋升', '定期体检', '五险一金'</td>\n",
       "      <td>职位描述：职位描述：1、收集、处理用户海量数据，挖掘用户行为特征，为产品、运营提供参考依据；...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>12500</td>\n",
       "      <td>500-2000人</td>\n",
       "      <td>C轮</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>'琶洲', '官洲'</td>\n",
       "      <td>六险一金,周末双休,营养工作餐,萌宠陪伴</td>\n",
       "      <td>广州</td>\n",
       "      <td>广州微咔世纪信息科技有限公司</td>\n",
       "      <td>大专</td>\n",
       "      <td>移动互联网,社交网络</td>\n",
       "      <td>'移动互联网', '社交', '数据运营'</td>\n",
       "      <td>'六险一金', '周末双休', '营养工作餐', '暖心下午茶'</td>\n",
       "      <td>职位描述：职位描述：1.负责对业务的专题进行统计分析，形成专题分析报告；2.基于业务理解，设...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>11500</td>\n",
       "      <td>50-150人</td>\n",
       "      <td>A轮</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>'中关村', '北京大学', '颐和园'</td>\n",
       "      <td>爱奇艺</td>\n",
       "      <td>北京</td>\n",
       "      <td>北京奇艺世纪科技有限公司</td>\n",
       "      <td>本科</td>\n",
       "      <td>广告营销,文化娱乐</td>\n",
       "      <td>'大数据', 'SPSS'</td>\n",
       "      <td>'绩效奖金', '五险一金', '交通补助', '带薪年假'</td>\n",
       "      <td>职位描述：岗位职责： 1、负责内容合作部产品及运营数据指标的搭建； 2、负责数据后台的完善与...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10000</td>\n",
       "      <td>2000人以上</td>\n",
       "      <td>上市公司</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                address             advantage city    company_name education  \\\n",
       "0  '科华北路', '桂溪', '四川大学'                 工作氛围好   成都  达疆网络科技（上海）有限公司        本科   \n",
       "1                  '未知'    六险二金,晋升通道,独当一面,话语权   北京  贝壳找房（北京）科技有限公司        本科   \n",
       "2   '中关村', '万泉河', '苏州街'                  五险一金   北京      杭州财米科技有限公司        不限   \n",
       "3            '琶洲', '官洲'  六险一金,周末双休,营养工作餐,萌宠陪伴   广州  广州微咔世纪信息科技有限公司        大专   \n",
       "4  '中关村', '北京大学', '颐和园'                   爱奇艺   北京    北京奇艺世纪科技有限公司        本科   \n",
       "\n",
       "     industry         industryLables                             label  \\\n",
       "0         O2O                           '年底双薪', '绩效奖金', '岗位晋升', '定期体检'   \n",
       "1   移动互联网,O2O            '大数据', '商业'    '股票期权', '带薪年假', '绩效奖金', '扁平管理'   \n",
       "2    移动互联网,金融                           '年底多薪', '岗位晋升', '定期体检', '五险一金'   \n",
       "3  移动互联网,社交网络  '移动互联网', '社交', '数据运营'  '六险一金', '周末双休', '营养工作餐', '暖心下午茶'   \n",
       "4   广告营销,文化娱乐          '大数据', 'SPSS'    '绩效奖金', '五险一金', '交通补助', '带薪年假'   \n",
       "\n",
       "                                     position_detail position_name salary  \\\n",
       "0  职位描述：工作职责：?1、负责新零售业务的数据分析工作，挖掘数据分析需求，制定并实施分析方案...         数据分析师  15000   \n",
       "1  职位描述：工作职责:方向一、经营分析/指标体系1. 参与公司核心策略的数据分析，基于策略逻辑...         数据分析师  32500   \n",
       "2  职位描述：职位描述：1、收集、处理用户海量数据，挖掘用户行为特征，为产品、运营提供参考依据；...         数据分析师  12500   \n",
       "3  职位描述：职位描述：1.负责对业务的专题进行统计分析，形成专题分析报告；2.基于业务理解，设...         数据分析师  11500   \n",
       "4  职位描述：岗位职责： 1、负责内容合作部产品及运营数据指标的搭建； 2、负责数据后台的完善与...         数据分析师  10000   \n",
       "\n",
       "        size  stage work_year  \n",
       "0    2000人以上  D轮及以上      1-3年  \n",
       "1    2000人以上     C轮     5-10年  \n",
       "2  500-2000人     C轮      1-3年  \n",
       "3    50-150人     A轮      1-3年  \n",
       "4    2000人以上   上市公司      1-3年  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 清洗过程模块化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>address</th>\n",
       "      <th>advantage</th>\n",
       "      <th>city</th>\n",
       "      <th>company_name</th>\n",
       "      <th>education</th>\n",
       "      <th>industry</th>\n",
       "      <th>industryLables</th>\n",
       "      <th>label</th>\n",
       "      <th>position_detail</th>\n",
       "      <th>position_name</th>\n",
       "      <th>salary</th>\n",
       "      <th>size</th>\n",
       "      <th>stage</th>\n",
       "      <th>work_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>'科华北路', '桂溪', '四川大学'</td>\n",
       "      <td>工作氛围好</td>\n",
       "      <td>成都</td>\n",
       "      <td>达疆网络科技（上海）有限公司</td>\n",
       "      <td>本科</td>\n",
       "      <td>O2O</td>\n",
       "      <td></td>\n",
       "      <td>'年底双薪', '绩效奖金', '岗位晋升', '定期体检'</td>\n",
       "      <td>职位描述：工作职责：?1、负责新零售业务的数据分析工作，挖掘数据分析需求，制定并实施分析方案...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>15000</td>\n",
       "      <td>2000人以上</td>\n",
       "      <td>D轮及以上</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'未知'</td>\n",
       "      <td>六险二金,晋升通道,独当一面,话语权</td>\n",
       "      <td>北京</td>\n",
       "      <td>贝壳找房（北京）科技有限公司</td>\n",
       "      <td>本科</td>\n",
       "      <td>移动互联网,O2O</td>\n",
       "      <td>'大数据', '商业'</td>\n",
       "      <td>'股票期权', '带薪年假', '绩效奖金', '扁平管理'</td>\n",
       "      <td>职位描述：工作职责:方向一、经营分析/指标体系1. 参与公司核心策略的数据分析，基于策略逻辑...</td>\n",
       "      <td>数据分析类</td>\n",
       "      <td>32500</td>\n",
       "      <td>2000人以上</td>\n",
       "      <td>C轮</td>\n",
       "      <td>5-10年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>'中关村', '万泉河', '苏州街'</td>\n",
       "      <td>五险一金</td>\n",
       "      <td>北京</td>\n",
       "      <td>杭州财米科技有限公司</td>\n",
       "      <td>不限</td>\n",
       "      <td>移动互联网,金融</td>\n",
       "      <td></td>\n",
       "      <td>'年底多薪', '岗位晋升', '定期体检', '五险一金'</td>\n",
       "      <td>职位描述：职位描述：1、收集、处理用户海量数据，挖掘用户行为特征，为产品、运营提供参考依据；...</td>\n",
       "      <td>数据分析师 (MJ000766)</td>\n",
       "      <td>12500</td>\n",
       "      <td>500-2000人</td>\n",
       "      <td>C轮</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>'琶洲', '官洲'</td>\n",
       "      <td>六险一金,周末双休,营养工作餐,萌宠陪伴</td>\n",
       "      <td>广州</td>\n",
       "      <td>广州微咔世纪信息科技有限公司</td>\n",
       "      <td>大专</td>\n",
       "      <td>移动互联网,社交网络</td>\n",
       "      <td>'移动互联网', '社交', '数据运营'</td>\n",
       "      <td>'六险一金', '周末双休', '营养工作餐', '暖心下午茶'</td>\n",
       "      <td>职位描述：职位描述：1.负责对业务的专题进行统计分析，形成专题分析报告；2.基于业务理解，设...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>11500</td>\n",
       "      <td>50-150人</td>\n",
       "      <td>A轮</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>'中关村', '北京大学', '颐和园'</td>\n",
       "      <td>爱奇艺</td>\n",
       "      <td>北京</td>\n",
       "      <td>北京奇艺世纪科技有限公司</td>\n",
       "      <td>本科</td>\n",
       "      <td>广告营销,文化娱乐</td>\n",
       "      <td>'大数据', 'SPSS'</td>\n",
       "      <td>'绩效奖金', '五险一金', '交通补助', '带薪年假'</td>\n",
       "      <td>职位描述：岗位职责： 1、负责内容合作部产品及运营数据指标的搭建； 2、负责数据后台的完善与...</td>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10000</td>\n",
       "      <td>2000人以上</td>\n",
       "      <td>上市公司</td>\n",
       "      <td>1-3年</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                address             advantage city    company_name education  \\\n",
       "0  '科华北路', '桂溪', '四川大学'                 工作氛围好   成都  达疆网络科技（上海）有限公司        本科   \n",
       "1                  '未知'    六险二金,晋升通道,独当一面,话语权   北京  贝壳找房（北京）科技有限公司        本科   \n",
       "2   '中关村', '万泉河', '苏州街'                  五险一金   北京      杭州财米科技有限公司        不限   \n",
       "3            '琶洲', '官洲'  六险一金,周末双休,营养工作餐,萌宠陪伴   广州  广州微咔世纪信息科技有限公司        大专   \n",
       "4  '中关村', '北京大学', '颐和园'                   爱奇艺   北京    北京奇艺世纪科技有限公司        本科   \n",
       "\n",
       "     industry         industryLables                             label  \\\n",
       "0         O2O                           '年底双薪', '绩效奖金', '岗位晋升', '定期体检'   \n",
       "1   移动互联网,O2O            '大数据', '商业'    '股票期权', '带薪年假', '绩效奖金', '扁平管理'   \n",
       "2    移动互联网,金融                           '年底多薪', '岗位晋升', '定期体检', '五险一金'   \n",
       "3  移动互联网,社交网络  '移动互联网', '社交', '数据运营'  '六险一金', '周末双休', '营养工作餐', '暖心下午茶'   \n",
       "4   广告营销,文化娱乐          '大数据', 'SPSS'    '绩效奖金', '五险一金', '交通补助', '带薪年假'   \n",
       "\n",
       "                                     position_detail     position_name salary  \\\n",
       "0  职位描述：工作职责：?1、负责新零售业务的数据分析工作，挖掘数据分析需求，制定并实施分析方案...             数据分析师  15000   \n",
       "1  职位描述：工作职责:方向一、经营分析/指标体系1. 参与公司核心策略的数据分析，基于策略逻辑...             数据分析类  32500   \n",
       "2  职位描述：职位描述：1、收集、处理用户海量数据，挖掘用户行为特征，为产品、运营提供参考依据；...  数据分析师 (MJ000766)  12500   \n",
       "3  职位描述：职位描述：1.负责对业务的专题进行统计分析，形成专题分析报告；2.基于业务理解，设...             数据分析师  11500   \n",
       "4  职位描述：岗位职责： 1、负责内容合作部产品及运营数据指标的搭建； 2、负责数据后台的完善与...             数据分析师  10000   \n",
       "\n",
       "        size  stage work_year  \n",
       "0    2000人以上  D轮及以上      1-3年  \n",
       "1    2000人以上     C轮     5-10年  \n",
       "2  500-2000人     C轮      1-3年  \n",
       "3    50-150人     A轮      1-3年  \n",
       "4    2000人以上   上市公司      1-3年  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import string\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "class data_clean(object):\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def get_data(self):\n",
    "        data1 = pd.read_csv('./data_analysis.csv', encoding='gbk')\n",
    "        data2 = pd.read_csv('./machine_learning.csv', encoding='gbk')\n",
    "        data3 = pd.read_csv('./data_mining.csv', encoding='gbk')\n",
    "        data4 = pd.read_csv('./deep_learning.csv', encoding='gbk')\n",
    "\n",
    "        data = pd.concat((pd.concat((pd.concat((data1, data2)), data3)), data4)).reset_index(drop=True)\n",
    "        return data\n",
    "\n",
    "    def clean_operation(self):\n",
    "        data = self.get_data()\n",
    "        data['address'] = data['address'].fillna(\"['未知']\")\n",
    "        for i, j in enumerate(data['address']):\n",
    "            j = j.replace('[', '').replace(']', '')\n",
    "            data['address'][i] = j\n",
    "            \n",
    "        for i, j in enumerate(data['salary']):\n",
    "            j = j.replace('k', '').replace('K', '').replace('以上', '-0')\n",
    "            j1 = int(j.split('-')[0])\n",
    "            j2 = int(j.split('-')[1])\n",
    "            j3 = 1/2 * (j1+j2)\n",
    "            data['salary'][i] = j3*1000\n",
    "            \n",
    "        for i, j in enumerate(data['industryLables']):\n",
    "            j = j.replace('[', '').replace(']', '')\n",
    "            data['industryLables'][i] = j\n",
    "            \n",
    "        for i, j in enumerate(data['label']):\n",
    "            j = j.replace('[', '').replace(']', '')\n",
    "            data['label'][i] = j\n",
    "         \n",
    "        data['position_detail'] = data['position_detail'].fillna('未知')\n",
    "        for i, j in enumerate(data['position_detail']):\n",
    "            j = j.replace('\\r', '')\n",
    "            data['position_detail'][i] = j\n",
    "            \n",
    "        return data\n",
    "    \n",
    "    \n",
    "opt = data_clean()\n",
    "data = opt.clean_operation()\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
